from urllib import parse

import re
import scrapy
from scrapy import Request
from scrapyNews.items import JobBoleArticleItem, ArticleItemLoader


class JobboleSpider(scrapy.Spider):
    name = "jobbole"
    allowed_domains = ["python.jobbole.com"]
    start_urls = ['http://python.jobbole.com/all-posts/']

    # name = "jobbole"
    # allowed_domains = ["blog.jobbole.com"]
    # start_urls = ['http://blog.jobbole.com/110287']

    def parse(self, response):
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            str1 = response.url
            str2 = post_url
            str = parse.urljoin(response.url, post_url)
            yield Request(url=parse.urljoin(response.url, post_url),
                          meta={"front_image_url": image_url}, callback=self.get_detail)
                          # meta={"front_image_url": image_url}, callback=self.parse_detail)
            next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        # if next_url:
            # yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)



    def parse_detail(self, response):
        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        article_item = JobBoleArticleItem()
        title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first("")
        create_date = response.xpath("//p[@class='entry-meta-hide-on-mobile']/text()").extract()[
            0].strip().replace("·", "").strip()
        praise_nums = \
        response.xpath("//span[contains(@class, 'vote-post-up')]/h10/text()").extract()[0]
        fav_nums = response.xpath("//span[contains(@class, 'bookmark-btn')]/text()").extract()[0]
        comment_nums = response.xpath("//a[@href='#article-comment']/span/text()").extract()[0]
        # match_re = re.match(".*?(\d+).*", comment_nums)
        # if match_re:
        #     comment_nums = match_re.group(1)
        content = response.xpath("//div[@class='entry']").extract()[0]
        tag_list = response.xpath("//p[@class='entry-meta-hide-on-mobile']/a/text()").extract()
        url=response.url
        article_item["title"] = title
        article_item["url"] = response.url
        article_item["create_date"] = create_date
        article_item["front_image_url"] = [front_image_url]
        article_item["praise_nums"] = praise_nums
        article_item["fav_nums"] = fav_nums
        article_item["comment_nums"] = comment_nums
        article_item["content"] = content
        yield article_item


    # 解析详情数据存入数据库
    def get_detail(self,response):
        article_item=JobBoleArticleItem()
        front_image_url=response.meta.get("front_image_url","")#文章封面
        item_loader=ArticleItemLoader(item=JobBoleArticleItem,response=response)
        item_loader.add_css("title",".entry-header h1::text")
        item_loader.add_value("url",response.url)
        # item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_css("create_date", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_value("front_image_url", [front_image_url])
        item_loader.add_css("praise_nums", ".vote-post-up h10::text")
        item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_nums", ".bookmark-btn::text")
        item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a::text")
        item_loader.add_css("content", "div.entry")

        article_item=item_loader.load_item()
        yield  article_item


